{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dressmaker - Easy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    }
   ],
   "source": [
    "import findspark\n",
    "import pandas as pd\n",
    "findspark.init()\n",
    "\n",
    "SVR = '192.168.31.31'\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql import Window\n",
    "\n",
    "sc = (SparkSession.builder.appName('app17-1') \n",
    "      .master(f'spark://{SVR}:7077') \n",
    "      .config('spark.sql.warehouse.dir', f'hdfs://{SVR}:9000/user/hive/warehouse') \n",
    "      .config('spark.cores.max', '4') \n",
    "      .config('spark.executor.instances', '1') \n",
    "      .config('spark.executor.cores', '2') \n",
    "      .config('spark.executor.memory', '10g') \n",
    "      .enableHiveSupport().getOrCreate())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "jmcust = sc.read.table('sqlzoo.jmcust')\n",
    "dressmaker = sc.read.table('sqlzoo.dressmaker')\n",
    "dress_order = sc.read.table('sqlzoo.dress_order')\n",
    "construction = sc.read.table('sqlzoo.construction')\n",
    "quantities = sc.read.table('sqlzoo.quantities')\n",
    "order_line = sc.read.table('sqlzoo.order_line')\n",
    "garment = sc.read.table('sqlzoo.garment')\n",
    "material = sc.read.table('sqlzoo.material')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.\n",
    "List the post code, order number, order date and garment descriptions for all items associated with Ms Brown."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post_code</th>\n",
       "      <th>order_no</th>\n",
       "      <th>order_date</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B2 5AB</td>\n",
       "      <td>9</td>\n",
       "      <td>2002-02-27</td>\n",
       "      <td>Shorts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B2 5AB</td>\n",
       "      <td>7</td>\n",
       "      <td>2002-02-21</td>\n",
       "      <td>Suntop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B2 5AB</td>\n",
       "      <td>7</td>\n",
       "      <td>2002-02-21</td>\n",
       "      <td>Sundress</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B2 5AB</td>\n",
       "      <td>7</td>\n",
       "      <td>2002-02-21</td>\n",
       "      <td>Short Skirt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   post_code  order_no  order_date           description\n",
       "0  B2 5AB            9  2002-02-27  Shorts              \n",
       "1  B2 5AB            7  2002-02-21  Suntop              \n",
       "2  B2 5AB            7  2002-02-21  Sundress            \n",
       "3  B2 5AB            7  2002-02-21  Short Skirt         "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1 = (jmcust.filter(trim(jmcust['c_name'])=='Ms Brown')\n",
    "      .join(dress_order, on=(jmcust['c_no']==dress_order['cust_no']))\n",
    "      .join(order_line, on=(dress_order['order_no']==order_line['order_ref']))\n",
    "      .join(garment, on=(order_line['ol_style']==garment['style_no']))\n",
    "      .withColumnRenamed('c_post_code', 'post_code')\n",
    "      .select('post_code', 'order_no', 'order_date', 'description'))\n",
    "t2 = (dressmaker.filter(trim(dressmaker['d_name'])=='Ms Brown')\n",
    "      .join(construction, on=(dressmaker['d_no']==construction['maker']))\n",
    "      .join(dress_order, on=(construction['order_ref']==dress_order['order_no']))\n",
    "      .join(order_line, on=(dress_order['order_no']==order_line['order_ref']))\n",
    "      .join(garment, on=(order_line['ol_style']==garment['style_no']))\n",
    "      .withColumnRenamed('d_post_code', 'post_code')\n",
    "      .select('post_code', 'order_no', 'order_date', 'description'))\n",
    "t1.union(t2).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.\n",
    "List the customer name, postal information, order date and order number of all orders that have been completed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>c_name</th>\n",
       "      <th>c_post_code</th>\n",
       "      <th>order_date</th>\n",
       "      <th>order_no</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ms White</td>\n",
       "      <td>E24 8PQ</td>\n",
       "      <td>2002-02-03</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ms White</td>\n",
       "      <td>E24 8PQ</td>\n",
       "      <td>2002-02-03</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ms White</td>\n",
       "      <td>E24 8PQ</td>\n",
       "      <td>2002-02-03</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mr Brass</td>\n",
       "      <td>FG24 9NM</td>\n",
       "      <td>2002-02-02</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mr Brass</td>\n",
       "      <td>FG24 9NM</td>\n",
       "      <td>2002-02-02</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Ms Muir</td>\n",
       "      <td>H2 7CV</td>\n",
       "      <td>2002-01-20</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Dr Green</td>\n",
       "      <td>SJ4 4WE</td>\n",
       "      <td>2002-01-11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Dr Green</td>\n",
       "      <td>SJ4 4WE</td>\n",
       "      <td>2002-01-11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Dr Green</td>\n",
       "      <td>SJ4 4WE</td>\n",
       "      <td>2002-01-11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Mrs Peacock</td>\n",
       "      <td>DD6 9NM</td>\n",
       "      <td>2002-01-10</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Mrs Peacock</td>\n",
       "      <td>DD6 9NM</td>\n",
       "      <td>2002-01-10</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  c_name c_post_code  order_date  order_no\n",
       "0   Ms White               E24 8PQ    2002-02-03         5\n",
       "1   Ms White               E24 8PQ    2002-02-03         5\n",
       "2   Ms White               E24 8PQ    2002-02-03         5\n",
       "3   Mr Brass               FG24 9NM   2002-02-02         4\n",
       "4   Mr Brass               FG24 9NM   2002-02-02         4\n",
       "5   Ms Muir                H2 7CV     2002-01-20         3\n",
       "6   Dr Green               SJ4 4WE    2002-01-11         2\n",
       "7   Dr Green               SJ4 4WE    2002-01-11         2\n",
       "8   Dr Green               SJ4 4WE    2002-01-11         2\n",
       "9   Mrs Peacock            DD6 9NM    2002-01-10         1\n",
       "10  Mrs Peacock            DD6 9NM    2002-01-10         1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(jmcust.join(dress_order.filter(dress_order['completed']=='Y'),\n",
    "             on=(jmcust['c_no']==dress_order['cust_no']))\n",
    " .join(order_line, on=(dress_order['order_no']==order_line['order_ref']))\n",
    " .select('c_name', 'c_post_code', 'order_date', 'order_no')\n",
    " .toPandas())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.\n",
    "Which garments have been made or are being made from 'red abstract' or 'blue abstract' coloured materials."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>style_no</th>\n",
       "      <th>description</th>\n",
       "      <th>colour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Trousers</td>\n",
       "      <td>Red Abstract</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Long Skirt</td>\n",
       "      <td>Blue Abstract</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Long Skirt</td>\n",
       "      <td>Red Abstract</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Short Skirt</td>\n",
       "      <td>Blue Abstract</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Sundress</td>\n",
       "      <td>Red Abstract</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   style_no           description                colour\n",
       "0         1  Trousers              Red Abstract        \n",
       "1         2  Long Skirt            Blue Abstract       \n",
       "2         2  Long Skirt            Red Abstract        \n",
       "3         4  Short Skirt           Blue Abstract       \n",
       "4         5  Sundress              Red Abstract        "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(garment.join(order_line, on=(garment['style_no']==order_line['ol_style']))\n",
    " .join(material.filter(lower(trim(material['colour'])).isin(\n",
    "     'red abstract', 'blue abstract')),\n",
    "     on=(order_line['ol_material']==material['material_no']))\n",
    " .select('style_no', 'description', 'colour')\n",
    " .toPandas())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.\n",
    "How many garments has each dressmaker constructed? You should give the number of garments and the name and postal information of each dressmaker."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>d_no</th>\n",
       "      <th>d_name</th>\n",
       "      <th>d_post_code</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>Mr Seam</td>\n",
       "      <td>H45 7YH</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>Mr Taylor</td>\n",
       "      <td>SH6 9RT</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>Miss Pins</td>\n",
       "      <td>B4 9BL</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Mr Needles</td>\n",
       "      <td>E12 6LG</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Mrs Hem</td>\n",
       "      <td>A12 6BC</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>Ms Sew</td>\n",
       "      <td>EF7 9KL</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>Miss Stitch</td>\n",
       "      <td>DF4 7HJ</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   d_no                d_name d_post_code  cnt\n",
       "0     5  Mr Seam                 H45 7YH    12\n",
       "1     6  Mr Taylor               SH6 9RT     7\n",
       "2     7  Miss Pins               B4 9BL     11\n",
       "3     3  Mr Needles              E12 6LG    12\n",
       "4     1  Mrs Hem                 A12 6BC    14\n",
       "5     4  Ms Sew                  EF7 9KL    12\n",
       "6     2  Miss Stitch             DF4 7HJ    20"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dressmaker.join(construction, on=(dressmaker['d_no']==construction['maker']))\n",
    " .join(dress_order, on=(construction['order_ref']==dress_order['order_no']))\n",
    " .join(order_line, on=(dress_order['order_no']==order_line['order_ref']))\n",
    " .join(garment, on=(order_line['ol_style']==garment['style_no']))\n",
    " .groupBy('d_no', 'd_name', 'd_post_code')\n",
    " .agg(count('line_no').alias('cnt'))\n",
    " .toPandas())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.\n",
    "Give the names of those dressmakers who have finished items made from silk for completed orders."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>d_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Miss Stitch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mrs Hem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mr Needles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ms Sew</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Miss Pins</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 d_name\n",
       "0  Miss Stitch         \n",
       "1  Mrs Hem             \n",
       "2  Mr Needles          \n",
       "3  Ms Sew              \n",
       "4  Miss Pins           "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dressmaker.join(construction, on=(dressmaker['d_no']==construction['maker']))\n",
    " .join(dress_order.filter(dress_order['completed']=='Y'), \n",
    "       on=(construction['order_ref']==dress_order['order_no']))\n",
    " .join(order_line, on=(dress_order['order_no']==order_line['order_ref']))\n",
    " .join(garment, on=(order_line['ol_style']==garment['style_no']))\n",
    " .join(material.filter(lower(trim(material['fabric']))=='silk'), \n",
    "        on=(order_line['ol_material']==material['material_no']))\n",
    " .select('d_name')\n",
    " .distinct()\n",
    " .toPandas())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
